/**
 * @公司名称: 上海恒生聚源数据服务有限公司
 * @部门    ：平台开发部
 * @工程名  : Kettle组件开发
 */
package org.jony.JProject.webFetch;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import net.sf.json.JSONObject;

import org.apache.commons.collections.ListUtils;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;

/**
 * @类说明:
 * @作者 ：Jony
 * @E-mail : lij@gildata.com
 * @创建日期: 2012-5-16上午9:20:24
 * @版本号 ：V.1.0
 */
public class CopyOfFetchForAshxAJAX2 {

	/**
	 * @功能介绍：
	 * @创建时间：2012-5-16上午9:20:24 by Jony
	 * @param args
	 */

	private final static int BUFFER = 1024;
	private final static String url = "http://www.zyctd.com/Prices/Ajax.ashx";
	private static CopyOfFetchForAshxAJAX2 ffaa = new CopyOfFetchForAshxAJAX2();

	public static void main(String[] args) {

		try {
			List<String> websites = FileUtils.readLines(new File(
					"d:\\websites.txt"));
//			for (String website : websites) {
//				ffaa.fetchbyWebSite(website);
//			}

			 ffaa.fetchbyWebSite("http://www.zyctd.com/exchange-prices-158-1-1.html?mbsid=15801");
		} catch (IOException ioe) {
			ioe.printStackTrace();
		}

	}

	private void fetchbyWebSite(String website) throws IOException {
		// 数据库连接信息
		String driverName = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; // 加载JDBC驱动
		String dbURL = "jdbc:sqlserver://172.16.53.183:1433; DatabaseName=DCDB"; // 连接服务器和数据库sample
		String userName = "sa"; // 默认用户名
		String userPwd = "123456";
		Connection conn = null;

		Map<String, String> webPara = null;
		Map<String, String> para = new HashMap<String, String>();

		// 根据链接地址取出网页参数
		webPara = JSONToMap(getStringByRegu(
				ffaa.getParaByWebsite(website, "gb2312"), "EDWIN,\\{(.*?)\\}")
				.replaceFirst("EDWIN,", ""));

		// 取出整个总共有多少页

		double count = Double.parseDouble(webPara.get("marketcount"));
		if (count == 0) {
			count = Double.parseDouble(webPara.get("proareacount"));
		}
		int pages = (int) Math.ceil(count
				/ Integer.parseInt(webPara.get("pagesize")));

		ArrayList<String> list = new ArrayList<String>();
		for (int i = 1; i <= pages; i++) {
			// 根据封装的参数将数据得到
			para.put("do", "getpagemarketprice");
			para.put("mbsid", webPara.get("mbsid"));
			para.put("size", webPara.get("pagesize"));
			para.put("curidx", String.valueOf(i));
			// 将结果缓存
			String content = getStringByRegu(ffaa.getContentByPara(para),
					"\\[\\[(.*?)\\]\\]");
			String data = StringUtils.remove(StringUtils.remove(content, "]]"),
					"日").replaceAll("[\u4e00-\u9fa5]", "-");
			String[] spl = StringUtils.splitByWholeSeparator(
					StringUtils.remove(data, "[["), "],[");
			list.addAll(Arrays.asList(spl));
		}

		// 封装数据为入库数据
		try {
			Class.forName(driverName);
			conn = DriverManager.getConnection(dbURL, userName, userPwd);
			ffaa.dataToDB(conn, website, list);
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		} catch (SQLException e) {
			e.printStackTrace();
		} finally {
			if (conn != null)
				try {
					conn.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
		}
		// 打印统计信息：
		StringBuilder info = new StringBuilder(website);
		info.append("一共披露数据有:");
		info.append(count);
		info.append("条，实际页数：");
		info.append(pages);
		info.append("，最终请求入库的条数：");
		info.append(list.size());
		System.out.println(System.currentTimeMillis()+"==="+info.toString());
		// FileUtils.writeStringToFile(new File("d:\\信息统计.txt"),
		// info.toString());
	}

	private void dataToDB(Connection conn, String website, List<String> list) {
		Statement st;
		try {
			st = conn.createStatement();
			for (String data : list) {
				String sql = "insert into webYYHYHQBJ(RQ,HZ,AG,CD,YL,WEBSITE) values("
						+ data + ",'" + website + "')";
				st.addBatch(sql);
			}
			st.executeBatch();
		} catch (SQLException e) {
			System.out.println(website + "在入库时发生错误，请处理！原因为："+e.getMessage());
			return;
		}

	}

	private static String getStringByRegu(String str, String regu) {
		Pattern p = Pattern.compile(regu);
		Matcher m = p.matcher(str);
		while (m.find()) {
			str = m.group();
		}
		return str;
	}

	private String getParaByWebsite(String website, String charset) {
		/*
		 * 使用 GetMethod 来访问一个 URL 对应的网页,实现步骤: 1:生成一个 HttpClinet 对象并设置相应的参数。
		 * 2:生成一个 GetMethod 对象并设置响应的参数。 3:用 HttpClinet 生成的对象来执行 GetMethod 生成的Get
		 * 方法。 4:处理响应状态码。 5:若响应正常，处理 HTTP 响应内容。 6:释放连接。
		 */

		HttpClient httpClient = new HttpClient();
		httpClient.getHttpConnectionManager().getParams()
				.setConnectionTimeout(5000);

		GetMethod getMethod = new GetMethod(website);
		getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 50000);
		getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
				new DefaultHttpMethodRetryHandler());

		String response = "";
		try {
			int statusCode = httpClient.executeMethod(getMethod);
			if (statusCode != HttpStatus.SC_OK) {
				System.err.println("网页获取失败！地址为：" + website
						+ getMethod.getStatusLine());
			}

			byte[] responseBody = getMethod.getResponseBody();// 读取为字节数组
			response = new String(responseBody, charset);

			// 读取为 InputStream，在网页内容数据量大时候推荐使用
			// InputStream response = getMethod.getResponseBodyAsStream();

		} catch (HttpException e) {
			System.out.println("Please check your provided http address!");
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			/* 释放连接 */
			getMethod.releaseConnection();
		}
		return response;
	}

	public String getContentByPara(Map para) {
		String result = null;

		HttpClient client = new HttpClient();
		PostMethod postMethod = null;
		postMethod = new PostMethod(url);

		// 将map中的参数传入("do","mbsid","curidx","size")
		Iterator it = para.entrySet().iterator();
		while (it.hasNext()) {
			Map.Entry pairs = (Map.Entry) it.next();
			postMethod.setParameter((String) pairs.getKey(),
					(String) pairs.getValue());
		}

		try {
			client.executeMethod(postMethod);
			result = postMethod.getResponseBodyAsString();
		} catch (HttpException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			postMethod.releaseConnection();
		}
		return result;
	}

	public static Map<String, String> JSONToMap(String s) {
		Map map = new HashMap<String, String>();
		JSONObject json = JSONObject.fromObject(s);
		Iterator keys = json.keys();
		while (keys.hasNext()) {
			String key = (String) keys.next();
			String value = json.get(key).toString();
			if (value.startsWith("{") && value.endsWith("}")) {
				map.put(key, JSONToMap(value));
			} else {
				map.put(key, value);
			}

		}
		return map;
	}

}
